#!/usr/bin/env python
# -*- coding:utf-8 -*-
# @FileName  :jingdian_list.py
# @Time      :2023/10/30 
# @Author    :CL
# @email     :1037654919@qq.com
import time

from bs4 import BeautifulSoup
import requests
import re
from place import get_place
from utils import mongo_manager,get_kuai_proxy
from retrying import retry
qunar_place = mongo_manager("qunar_place",db = 'lvyou')
qunar_sight_info = mongo_manager("qunar_sight_info",db = 'lvyou')
headers = {
    "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7",
    "Accept-Language": "zh-CN,zh;q=0.9",
    "Cache-Control": "no-cache",
    "Pragma": "no-cache",
    "Proxy-Connection": "keep-alive",
    "Upgrade-Insecure-Requests": "1",
    "User-Agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36"
}
cookies = {
    "QN1": "0000c180306c57ae8cf8d525",
    "QN300": "s%3Dgoogle",
    "QN99": "8159",
    "QunarGlobal": "10.76.34.188_476f0ecd_18b7f08d2d0_-316c|1698646943873",
    "qunar-assist": "{%22version%22:%2220211215173359.925%22%2C%22show%22:false%2C%22audio%22:false%2C%22speed%22:%22middle%22%2C%22zomm%22:1%2C%22cursor%22:false%2C%22pointer%22:false%2C%22bigtext%22:false%2C%22overead%22:false%2C%22readscreen%22:false%2C%22theme%22:%22default%22}",
    "QN205": "s%3Dgoogle",
    "QN277": "s%3Dgoogle",
    "csrfToken": "bLP49THyP6qyD4ZpvJopVkZIYwMJQoeX",
    "_i": "DFiEuZkoc2SDZ7nLtuc9nvvIcDSw",
    "QN269": "B1AEDAD076EC11EEBD98FA163EC565DF",
    "QN601": "1844dcac9a8193dff4a2197440799d4c",
    "QN163": "0",
    "fid": "b781e0b7-f523-4a6c-b67d-c9266a2dc911",
    "QN48": "tc_f450c1fd5a4f469f_18b7f3f743c_83bc",
    "ariaDefaultTheme": "null",
    "QN57": "16986471185280.9089655108923953",
    "QN58": "1698647118527%7C1698647185294%7C4",
    "QN5": "qunar_djmp_gnmdd_%E6%AD%A6%E6%B1%89",
    "Hm_lvt_c56a2b5278263aa647778d304009eafc": "1698647830",
    "viewpoi": "9209153|715901",
    "_vi": "VJWhlOQ7YuFwbQReSPhbup_PLMZLcz_QTSQ35JnkW6FMFuKJUB7IZUrhclofUABatR0PnK9LCKx0FEpuQlYMr6VUS9LfNx7Pkvac7bg24Pmx1wEaNFKsrfUoBZNSysW8_JVIxFbN6CfJnnnwMmC_LYN4L9zhNdo-L-9rbuKm8fDY",
    "viewdist": "300085-1|299782-3",
    "uld": "1-299782-4-1698652081|1-300085-1-1698651675|1-368750-2-1698647842",
    "JSESSIONID": "DD750E74AE55FCFF91076E83A13CD7E2",
    "Hm_lpvt_c56a2b5278263aa647778d304009eafc": "1698652082",
    "SECKEY_ABVK": "6VSzwQie0IIBLBlsiSTzPRToZ1xJ79WT2UdXhpOoG8w%3D",
    "QN271": "8545e4b1-bd9a-406a-935a-e1131b74140d",
    "BMAP_SECKEY": "NmQ3_dWPJQCKo0At1o15eEFzEwjZD5iBVIwUxO_ff8--Rd7zmPFDqb5mLfefk2_NHLac-MzrWFjv7YTjJr2VIy609CZSLUIqa0uMG4-yrU91YbVk-5P7pwHrIiD4XnY6nNSgfYJ1z2tCmsPcWAuE36MzbzpF9T8ebuh7HpUNHgqX6SDmHOzVYrE3Gl97r6y8",
    "QN267": "028366905eb65af57"
}
@retry(stop_max_attempt_number = 3)
def get_jingdian_list(url):
    response = requests.get(url, headers=headers, proxies=get_kuai_proxy(), cookies=cookies, timeout=10, verify=False)

    if response.status_code == 200:
        return response.text
if  __name__ == "__main__":
    # lists = get_place()
    # reslut = {
    #     "_id": "这是一条注释信息",
    #     "city": "景点所在的城市",
    #     "url": "景点所在的网页链接",
    #     "chinese_name": "景点中文名",
    #     "english_name": "景点英文名",
    #     "tit_num": "景点在所在城市的网页搜索排名",
    #     "strategy": "景点的攻略数",
    #     "comment": "景点的评论数",
    #     "href": "景点的网址 作为数据的_id,",
    #     "comment_sum": "多少%去过所在城市的驴友来过这里",
    #     "ranking_sum": "景点在所在城市的排名",
    #     "desbox": "描述。"
    # }
    # qunar_sight_info.insertOne(reslut)
    seeds =qunar_place.findAll({'count':{'$exists':True},'status':None})

    for seed in seeds:
        print(seed)
        count = int(seed['count'])
        for page in range(1,int(count/10)+1):

            url =seed['href'] + f'-jingdian-1-{page}'
            print(url)
            response = get_jingdian_list(url)
            if response:
                soup = BeautifulSoup(response, 'html.parser')
                try:
                    datas = soup.find('div', class_='listbox').find_all('li', class_='item')
                    for data in datas:
                        try:
                            result = {}
                            result['city'] =seed['city']
                            result['url'] = url
                            name = data.find('a').find('span',class_= 'cn_tit').text
                            # 使用正则表达式去除英文部分
                            result['chinese_name'] = re.sub(r'[A-Za-z\s]+', '', name)
                            result['english_name'] = data.find('a').find('span',class_= 'en_tit').text

                            tit_num = data.find('span', class_='tit_num').get_text()
                            # print(tit_num)
                            result['tit_num'] = tit_num #景点排名
                            icon_strategy = data.find('div', class_='strategy_sum').get_text()
                            # print(icon_strategy)
                            result['strategy'] = icon_strategy # 攻略
                            icon_comment = data.find('div', class_='comment_sum').get_text()
                            # print(icon_comment)
                            result['comment'] = icon_comment  #点评

                            # print(result['chinese_name'],data.find('a').get('href'))
                            result['href'] = data.find('a').get('href') # 连接
                            result['_id'] = result['href']
                            desc = data.find_all('div', class_='txtbox clrfix')[-1]
                            comment_sum = desc.find('span', class_='comment_sum').get_text()
                            # print(comment_sum)
                            result['comment_sum'] = comment_sum  #城市游客占比

                            ranking_sum = desc.find('span', class_='ranking_sum').get_text()
                            # print(ranking_sum)
                            result['ranking_sum'] = ranking_sum  #城市景点排名

                            desbox = desc.find('div', class_='desbox').get_text()
                            result['desbox'] = desbox  #景点描述

                            # print(desbox)

                            try:
                                qunar_sight_info.insertOne(result)
                            except BaseException as e:
                                print(e)
                                qunar_sight_info.updateOne({"_id":result["_id"]},result)
                        except BaseException as e:
                            print(e)
                except BaseException as e:
                    print(e)
            # break
        seed['status'] = 'success'
        qunar_place.updateOne({"_id":seed["_id"]},seed)
        time.sleep(5)


    qunar_place.close()
    qunar_sight_info.close()
